use Encode;
$|=1;
$Seperator=" ";
$Only_Remain_Words=0;
$One_Line_One_Sentence=0;
$tf_counting=1;

@ARGV == 1 or die "Usage:Segment_file.pl filename\n";
print "Loading Dictionary...\n";

open(FileIn,"word.txt");
$MaxLen=0;
while($Line=<FileIn>)
{
    chomp $Line;  
    $Line = decode("gbk",$Line);
    $MapDict{$Line}=length($Line);
    if ( length($Line) > $MaxLen  )
    {
        $MaxLen=length($Line);
	}
}
close(FileIn);

open IN,$ARGV[0] or die;
open OUT,">$ARGV[0].seg" or die;
$line_counter=0;
while($Line=<IN>)
{
    $line_counter++;
    chomp;
    if(not $One_Line_One_Sentence)
    {    
        while($NL=<IN>)   
        {
            $line_counter++;
            chomp $NL;
            if($NL =~ /^\s*$/)
            {
                last;
            }
            else
            {
                $Line.=$NL;
            }            
        }
    } 
    $Line = decode("gbk",$Line);
    $Result=Segment($Line);
    print OUT encode("gbk",$Result),"\n";
    print "$line_counter line processed\n" if($line_counter%100==0);
    if($tf_counting)
    {
        $tf{$_}++ foreach(split($Seperator,$Result));
    }
}
close IN;
close OUT;
print "segmented file is in $ARGV[0].seg\n";
if($tf_counting)
{
    open TF,">$ARGV[0].tf" or die;
    print TF encode("gbk",$_),":$tf{$_}\n" foreach (sort{$tf{$b}<=>$tf{$a}} keys %tf);
    close TF;
    print "term frequency file is in $ARGV[0].tf\n";
}


sub Segment
{
    my ($Input)=@_;
    my $Segemted=();
	#preprocessing	
    $Input =~ s/([a-zA-Z0-9_]+)/ $1 /g;
    if($Only_Remain_Words)
    {
        $Input =~ s/[^\w]/ /g;
    }
    else
    {
       $Input =~ s/([^\w])/ $1 /g;
    }
    $Input =~ s/\s+/ /g;
    @Lines = split(' ',$Input);
    $Result="";

    foreach(@Lines)
    {
        my $Remained=$_;
        if($Remained =~ /^[a-zA-Z0-9_]+$/ or length($Remained)<2)
        {
            $Result.="$Remained$Seperator";
            next;
        }
		while ( length($Remained) > 0 )
        {
            $Match=0;
            for($i=$MaxLen;$i>1;$i-- )
		    {
			      $MatchString=substr($Remained,0,$i);
			      if ( defined $MapDict{$MatchString} )
			      {
				        $Result.="$MatchString$Seperator";
				        $Remained=substr($Remained,$i,length($Remained)-$i);
				        $Match=1;
				        last;
			      }
            }
            if ( $Match == 0 )
            {
			    $Result.=substr($Remained,0,1);
			    $Result.=$Seperator;
			    $Remained=substr($Remained,1,length($Remained)-1);
            }
        }
    }
	return $Result;
}